import scrapy

class nextSpider(scrapy.Spider):
    name = "nextSpider"
    allowed_domains = ["lab.scrapyd.cn"]
    start_urls = ['http://lab.scrapyd.cn']

    def parse(self,response):
        mingyan = response.css('div.quote') # extract all quote from homepage,and save to var mingyan

        for v in mingyan: # loop get every quote's :quotes content,author,tags

            text = v.css('.text::text').extract_first() # extract quotes
            author = v.css('.author::text').extract_first() # extract author
            tags = v.css('.tags .tag::text').extract() # extract tags
            tags = ','.join(tags) # transfer thr array to string

            """
            next go write file,every personage's quotes save to a txt file
            """

            filename = '%s-语录.txt' % author # define filename,like : 木心-语录.txt

            with open(filename,"a+") as f: # different people;s quotes save to different txt file respectively,"a+" means add to 
                f.write(text)
                f.write('\n') # '\n' means next lien
                f.write('标签: ' + tags)
                f.write('\n-----------------------\n')
                f.close()

        # above si last course's content
        # nexty we need to judge if next page is exist,if exist
        # we shoudl keep submitint to parse operate ,whats is essential is how scrapy realise submit url 

        next_page = response.css('li.next a::attr(href)').extract_first() # css selector extract next page's url

        if next_page is not None: # judge if has next page
            """
            if it is a relative path,like : /page/1
                urljoin can transfer it to absolute path, in other words ,add our domain to id
                at last next_page is:http://lab.scrapyd.cn/page/2/
            """

            next_page = response.urljoin(next_page)

            """
            next it's the secrect to crawl the next page:
            scray supports us a fnciton like this: scrapy.Request()
            this funciton has many other vars,and we would taolk about it next,here we just use 2 vars
            one is:we keep crawling url(next_page),it's next page url,it canbe content page
            anather one:which function we can submit our url to crawl (callback=self.parse),here it's parse ,and it's the currnet function
            of course,we can write another function,like : content page,special handle content page data
            through a function like this,naext page url have been submited to parse,then we can constantly crawl,util no next page exist
            """

            yield scrapy.Request(next_page,callback=self.parse) # so this parse is a recurive function